﻿using System.Collections.Generic;

namespace ToyundaPhone.Helpers
{
	public class TextTokenizer : ToyundaPhone.Database.IWordExtractor, ITextSplitter
	{
		static System.Text.RegularExpressions.Regex wordExtractor = new System.Text.RegularExpressions.Regex(@"\w+", System.Text.RegularExpressions.RegexOptions.Compiled);

		#region IWordExtractor Members

		public IEnumerable<string> GetWords(string source)
		{
			List<string> words = new List<string>();
			foreach (System.Text.RegularExpressions.Match wordMatch in wordExtractor.Matches(source))
			{
				string lcWord = wordMatch.Value.ToLowerInvariant();
				if (words.Contains(lcWord))
				{
					continue;
				}
				words.Add(lcWord);
			}
			return words;
		}

		#endregion

		#region ITextSplitter Members

		public IEnumerable<TextSplitterToken> GetTokens(string text)
		{
			List<TextSplitterToken> tokens = new List<TextSplitterToken>();
			if (!string.IsNullOrEmpty(text))
			{
				int lastEnd = 0;
				foreach (System.Text.RegularExpressions.Match match in wordExtractor.Matches(text))
				{
					if (match.Index > lastEnd)
					{
						tokens.Add(new TextSplitterToken(lastEnd, false, text.Substring(lastEnd, match.Index - lastEnd)));
					}
					tokens.Add(new TextSplitterToken(match.Index, true, match.Value));
					lastEnd = match.Index + match.Length;
				}
				if (lastEnd < text.Length)
				{
					tokens.Add(new TextSplitterToken(lastEnd, false, text.Substring(lastEnd)));
				}
			}
			return tokens;
		}

		#endregion
	}
}
